{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "TPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/LC1332/Chat-Haruhi-Suzumiya/blob/main/characters/novel_collecting/%E5%AF%B9%E8%AF%9D%E5%92%8C%E6%91%98%E8%A6%81%E9%87%8D%E7%BB%84%E5%B0%8F%E8%AF%B4_%E4%B8%A4%E7%A7%8D%E6%96%B9%E5%BC%8F.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# 由对话和摘要重组小说\n",
        "## created by 李鲁鲁 and 米唯实\n",
        "\n",
        "- [Done] 尝试定位summrization在原文本中的位置\n",
        "- [Done] 尝试定位dialogue在原文本中的位置\n",
        "- [Done] 将定位后的文本重建为jsonl文件\n",
        "- [Done] 导入切分好的chunk"
      ],
      "metadata": {
        "id": "G2mnO6pbdyJB"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 载入需要的库"
      ],
      "metadata": {
        "id": "DygZQSoSSayj"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "qx_UEiFnSEFL",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "24d17208-d684-481e-aa9c-4d37eb44d63e"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m8.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m76.5/76.5 kB\u001b[0m \u001b[31m7.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.7/1.7 MB\u001b[0m \u001b[31m14.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.4/49.4 kB\u001b[0m \u001b[31m3.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Building wheel for google-search-results (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m76.5/76.5 kB\u001b[0m \u001b[31m4.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h"
          ]
        }
      ],
      "source": [
        "!pip -q install langchain openai google-search-results tiktoken\n",
        "!pip -q install kor markdownify\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip -q install transformers"
      ],
      "metadata": {
        "id": "CyLw-T_Xatdk",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d4678ded-61c8-4986-b310-a0c56db94bfd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.5/7.5 MB\u001b[0m \u001b[31m16.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m268.8/268.8 kB\u001b[0m \u001b[31m18.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m34.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m44.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import os\n",
        "import copy\n",
        "import json"
      ],
      "metadata": {
        "id": "LRocJU3un-ey"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 配置参数"
      ],
      "metadata": {
        "id": "0jYYn3kH7EmW"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# mount google drive\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d71c80af-b7be-4a64-c622-8614a9bdd59f",
        "id": "WjPEofM97E79"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "以下必须修改"
      ],
      "metadata": {
        "id": "RM0BOiF4S7AI"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# 储存txt和jsonl的文件夹路径。如需修改，请与下方自动化循环保持一致\n",
        "\n",
        "save_folder_path =  \"/content/drive/MyDrive/reorganized_story_shediaoyingxiongzhuan\"\n",
        "\n",
        "# chunk所在文件夹，请以_raw结尾\n",
        "folder_path = f\"/content/drive/MyDrive/shediaoyingxiongzhuan_extract\"\n"
      ],
      "metadata": {
        "id": "PkRGgOa37H1K"
      },
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "以下可选择性修改"
      ],
      "metadata": {
        "id": "CFZrCauHS1K1"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# 故事名字，默认为_raw之前的名字\n",
        "story_name_en = os.path.basename(folder_path).split(\"_\")[0]\n",
        "\n",
        "# 测试ID\n",
        "id = 200\n",
        "\n",
        "# 默认的保存路径\n",
        "save_jsonl_path = f\"/content/drive/MyDrive/reorganized_story_{story_name_en}/reorganized_{story_name_en}.jsonl\"\n",
        "save_txt_path = f\"/content/drive/MyDrive/reorganized_story_{story_name_en}/reorganized_{story_name_en}.txt\"\n",
        "\n",
        "# 默认抽取出的dialogue和summary文件位置/如果有不同请在此处和底部自动程序中修改\n",
        "save_folder = f\"/content/drive/MyDrive/{story_name_en}_extract\"\n",
        "\n",
        "dialoge_file = os.path.join(save_folder, f\"{id}_dialogue.txt\")\n",
        "summarzie_file = os.path.join(save_folder, f\"{id}_sum.txt\")"
      ],
      "metadata": {
        "id": "0M0DCUMhSx8P"
      },
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 通过切分好的chunk"
      ],
      "metadata": {
        "id": "OqhmONBrwFHt"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# folder_path = f\"/content/drive/MyDrive/shediaoyingxiongzhuan_extract\"\n",
        "# chunk_text = []\n",
        "\n",
        "import os\n",
        "\n",
        "chunk_text = [\"\"] * (sum([1 for file_name in os.listdir(folder_path) if file_name.endswith(\"_sum.txt\")]))\n",
        "\n",
        "# 遍历文件夹中的文件\n",
        "for file_name in os.listdir(folder_path):\n",
        "    if file_name.endswith(\"_raw.txt\"):\n",
        "        file_path = os.path.join(folder_path, file_name)\n",
        "\n",
        "        # 提取文件名中的 i\n",
        "        i = int(file_name.split('_')[0])\n",
        "\n",
        "        # 打开文件并读取内容\n",
        "        with open(file_path, 'r') as file:\n",
        "            text = file.read()\n",
        "\n",
        "        # 将文件内容添加到列表中，按 i 的顺序插入到正确位置\n",
        "        chunk_text[i] = text\n",
        "\n",
        "print(len(chunk_text))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JNYhEDXdwEO9",
        "outputId": "1cc00b84-0841-46f0-8b7e-29b0336fd961"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "929\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 导入抽取出来的dialogue和sum文件，成为一个list"
      ],
      "metadata": {
        "id": "rsZatxC3Wdtl"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# 如果需要先切分，需要运行\n",
        "# mount google drive\n",
        "# from google.colab import drive\n",
        "# drive.mount('/content/drive')"
      ],
      "metadata": {
        "id": "XXT0-k-oWozt"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "保存的路径可以修改"
      ],
      "metadata": {
        "id": "hfHZDRqFVULn"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 手动导入ID，继续向下运行以测试，最后根据ID循环"
      ],
      "metadata": {
        "id": "aJ_AeX7hLy1H"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "raw_text = chunk_text[ id ]\n",
        "\n",
        "\n",
        "chunk_sum = []\n",
        "unique_chunk_sum = []\n",
        "# 给定summarzie_file = os.path.join(save_folder, f\"{id}_sum.txt\")\n",
        "\n",
        "# 先检查这个文件是否存在\n",
        "\n",
        "# 然后使用utf-8编码打开，检查每一行，如果strip后，行首是'-'，则把后面的字符串append到一个list chunk_sum中\n",
        "\n",
        "# 请用python为我实现\n",
        "if os.path.exists(summarzie_file):\n",
        "    with open(summarzie_file, 'r', encoding='utf-8') as f:\n",
        "        for line in f:\n",
        "            if line.strip().startswith('-'):\n",
        "                chunk_sum.append(line.strip()[1:].strip())\n",
        "if os.path.exists(dialoge_file):\n",
        "  with open(dialoge_file, encoding='utf-8') as f:\n",
        "    dialogues = []\n",
        "    for line in f:\n",
        "        dialogue = json.loads(line)\n",
        "        dialogues.append(dialogue)\n",
        "\n",
        "unique_dialogue = []\n",
        "for item in dialogues:\n",
        "    if item not in unique_dialogue:\n",
        "        unique_dialogue.append(item)\n",
        "dia_texts = [data['dialogue'] for data in unique_dialogue]\n",
        "for item in chunk_sum:\n",
        "    if item not in unique_chunk_sum:\n",
        "        unique_chunk_sum.append(item)\n",
        "\n",
        "chunk_sum = unique_chunk_sum\n",
        "dialogues = unique_dialogue\n",
        "print(dialogues)\n",
        "print(dia_texts)\n",
        "print(chunk_sum)"
      ],
      "metadata": {
        "id": "s_8LHrEwUoKR",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "60a35565-f72a-49da-a8fe-4dc46b989169"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[{'role': '郭靖', 'dialogue': '郭靖一踏进房，便觉药气冲鼻，又见桌上、榻上、地下，到处放满了诸般药材，以及大大小小的瓶儿、罐儿、缸儿、钵儿，看来梁子翁喜爱调弄丹药，虽在客中，也不放下这些家伙。那小童显也熟习药性，取了四味药，用白纸分别包了，交给简管家。'}, {'role': '郭靖', 'dialogue': '郭靖伸手接过，转身出房。他药已到手，不再看住简管家。不料这管家甚是狡猾，出房时故意落后，待郭靖与那小童一出门，立时将门关上，撑上门闩，大声叫喊：“有贼啊，有贼啊”郭靖一怔，转身推门，那门甚是坚实，一时推之不开。那青衣童子年纪虽小，却机怜异常，听得简管家叫喊，知道不妙，乘郭靖使力推门之际，夹手抢过他手中那四包药，往旁边池塘中一丢。郭靖击出两掌，居然都给他闪避开去。'}, {'role': '郭靖', 'dialogue': '郭靖又惊又怒，双掌按在门上，运起内力，喀喇一响，门闩立时崩断。'}, {'role': '郭靖', 'dialogue': '他抢进门去，一拳击在简管家下颚之上，颚骨登时碎裂，哪里还能做声幸好梁子翁性喜僻静，居处指定要与别的房舍远离，那简管家这几下叫唤，倒无旁人听到。'}, {'role': '郭靖', 'dialogue': '他回身出门，见那童子已奔在数丈之外，急忙提气纵身，霎时间已追到身后，伸手往他后领抓落。那童子听得脑后风响，身子一挫，右腿横扫，身手竟自不弱。郭靖知道只要给他声张出来，不但药物不能得手，而且黄蓉与自己尚有性命之忧，下手更不容情，钩、拿、抓、打，招招是分筋错骨手的狠辣家数。'}, {'role': '郭靖', 'dialogue': '那童子跟着梁子翁，到处受人尊敬，从未遇过强敌，这时不觉心慌意乱，脸上连中了两拳。郭靖乘势直上，拍的一记，又在他天灵盖上击了一掌，那童子立时昏晕过去。郭靖提足将他拨入路旁草丛，回进房去，打火点亮蜡烛，见那简管家倒在地下，兀自昏晕。'}, {'role': '郭靖', 'dialogue': '郭靖暗骂自己胡涂：“那童儿刚才从哪四个瓶罐里取药，我可全没留意，现今怎知这四味药放在哪里”但见瓶罐上面画的都是些弯弯曲曲的符号，竟无一个文字，心下好生为难：“记得他是站在这里拿的，我且把这个角落里的数十罐药每样都拿些，回头请王道长选出来就是。”取过一叠白纸，每样药材都包了一包，生怕刚才简管家叫喊时被人听见，心里一急，包得更加慢了。'}, {'role': '郭靖', 'dialogue': '好容易在每个药瓶中都取了药包好，揣在怀里，大功告成，心下欢喜，回过身来，不提防手肘在旁边的大竹篓上一撞。那竹篓横跌翻倒，盖子落下，蓦地呼噜一声，窜出一条殷红如血的大蛇，猛向他脸上扑来。'}, {'role': '郭靖', 'dialogue': '郭靖大吃一惊，急忙向后纵开，只见那蛇身子有小碗粗细，半身尚在篓中，不知其长几何，最怪的是通体朱红，蛇头忽伸忽缩，蛇口中伸出一条分叉的舌头，不住向他摇动。'}]\n",
            "['郭靖一踏进房，便觉药气冲鼻，又见桌上、榻上、地下，到处放满了诸般药材，以及大大小小的瓶儿、罐儿、缸儿、钵儿，看来梁子翁喜爱调弄丹药，虽在客中，也不放下这些家伙。那小童显也熟习药性，取了四味药，用白纸分别包了，交给简管家。', '郭靖伸手接过，转身出房。他药已到手，不再看住简管家。不料这管家甚是狡猾，出房时故意落后，待郭靖与那小童一出门，立时将门关上，撑上门闩，大声叫喊：“有贼啊，有贼啊”郭靖一怔，转身推门，那门甚是坚实，一时推之不开。那青衣童子年纪虽小，却机怜异常，听得简管家叫喊，知道不妙，乘郭靖使力推门之际，夹手抢过他手中那四包药，往旁边池塘中一丢。郭靖击出两掌，居然都给他闪避开去。', '郭靖又惊又怒，双掌按在门上，运起内力，喀喇一响，门闩立时崩断。', '他抢进门去，一拳击在简管家下颚之上，颚骨登时碎裂，哪里还能做声幸好梁子翁性喜僻静，居处指定要与别的房舍远离，那简管家这几下叫唤，倒无旁人听到。', '他回身出门，见那童子已奔在数丈之外，急忙提气纵身，霎时间已追到身后，伸手往他后领抓落。那童子听得脑后风响，身子一挫，右腿横扫，身手竟自不弱。郭靖知道只要给他声张出来，不但药物不能得手，而且黄蓉与自己尚有性命之忧，下手更不容情，钩、拿、抓、打，招招是分筋错骨手的狠辣家数。', '那童子跟着梁子翁，到处受人尊敬，从未遇过强敌，这时不觉心慌意乱，脸上连中了两拳。郭靖乘势直上，拍的一记，又在他天灵盖上击了一掌，那童子立时昏晕过去。郭靖提足将他拨入路旁草丛，回进房去，打火点亮蜡烛，见那简管家倒在地下，兀自昏晕。', '郭靖暗骂自己胡涂：“那童儿刚才从哪四个瓶罐里取药，我可全没留意，现今怎知这四味药放在哪里”但见瓶罐上面画的都是些弯弯曲曲的符号，竟无一个文字，心下好生为难：“记得他是站在这里拿的，我且把这个角落里的数十罐药每样都拿些，回头请王道长选出来就是。”取过一叠白纸，每样药材都包了一包，生怕刚才简管家叫喊时被人听见，心里一急，包得更加慢了。', '好容易在每个药瓶中都取了药包好，揣在怀里，大功告成，心下欢喜，回过身来，不提防手肘在旁边的大竹篓上一撞。那竹篓横跌翻倒，盖子落下，蓦地呼噜一声，窜出一条殷红如血的大蛇，猛向他脸上扑来。', '郭靖大吃一惊，急忙向后纵开，只见那蛇身子有小碗粗细，半身尚在篓中，不知其长几何，最怪的是通体朱红，蛇头忽伸忽缩，蛇口中伸出一条分叉的舌头，不住向他摇动。']\n",
            "['郭靖进入房间，发现满满都是药材和药瓶', '简管家故意将门关上，喊贼引起郭靖的注意', '郭靖推门不开，青衣童子夺走了药包并扔进池塘', '郭靖打破门闩，击败简管家', '郭靖追赶青衣童子，展开激烈的战斗', '郭靖击晕童子，将其藏在草丛中', '郭靖包好药材，不小心撞倒竹篓，释放出一条红色蛇']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 给定长文本raw_text。\n",
        "\n",
        "# 使用换行符\\n或者。 来对这个字符串进行切割，忽略掉strip之后是空的子字符串\n",
        "\n",
        "# 将每一段话的起点位置存储在一个list of int , starts中\n",
        "\n",
        "# 将每一段话的结束位置存储在一个list of int , ends中\n",
        "\n",
        "# 并且将每一个子字符串的存储在一个list of str, lines中\n",
        "\n",
        "def divide_raw2lines(raw_text):\n",
        "  previous_str = ''\n",
        "  starts = []\n",
        "  ends = []\n",
        "  lines = []\n",
        "  for i in range(len(raw_text)):\n",
        "      previous_str += raw_text[i]\n",
        "      if raw_text[i] in ('\\n','。'):\n",
        "          strip_str = previous_str.strip(' \"“”\\r\\n')\n",
        "          if len(strip_str)>0:\n",
        "              lines.append(strip_str)\n",
        "              starts.append(i - len(strip_str))\n",
        "              ends.append(i)\n",
        "          previous_str = ''\n",
        "      else:\n",
        "          pass\n",
        "  return lines , starts , ends\n",
        "\n",
        "\n",
        "lines, starts, ends = divide_raw2lines(raw_text)\n",
        "\n",
        "print(len(lines))\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TWhq998rX4sT",
        "outputId": "7951ccf8-c9a6-475f-ca4c-1cccf22d99b4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "20\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(lines)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aepXbeJ-g8Fw",
        "outputId": "8561fc35-2810-4458-bb97-df6cd7757dd3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['郭靖一踏进房，便觉药气冲鼻，又见桌上、榻上、地下，到处放满了诸般药材，以及大大小小的瓶儿、罐儿、缸儿、钵儿，看来梁子翁喜爱调弄丹药，虽在客中，也不放下这些家伙。', '那小童显也熟习药性，取了四味药，用白纸分别包了，交给简管家。', '郭靖伸手接过，转身出房。', '他药已到手，不再看住简管家。', '不料这管家甚是狡猾，出房时故意落后，待郭靖与那小童一出门，立时将门关上，撑上门闩，大声叫喊：“有贼啊，有贼啊”郭靖一怔，转身推门，那门甚是坚实，一时推之不开。', '那青衣童子年纪虽小，却机怜异常，听得简管家叫喊，知道不妙，乘郭靖使力推门之际，夹手抢过他手中那四包药，往旁边池塘中一丢。', '郭靖击出两掌，居然都给他闪避开去。', '郭靖又惊又怒，双掌按在门上，运起内力，喀喇一响，门闩立时崩断。', '他抢进门去，一拳击在简管家下颚之上，颚骨登时碎裂，哪里还能做声幸好梁子翁性喜僻静，居处指定要与别的房舍远离，那简管家这几下叫唤，倒无旁人听到。', '他回身出门，见那童子已奔在数丈之外，急忙提气纵身，霎时间已追到身后，伸手往他后领抓落。', '那童子听得脑后风响，身子一挫，右腿横扫，身手竟自不弱。', '郭靖知道只要给他声张出来，不但药物不能得手，而且黄蓉与自己尚有性命之忧，下手更不容情，钩、拿、抓、打，招招是分筋错骨手的狠辣家数。', '那童子跟着梁子翁，到处受人尊敬，从未遇过强敌，这时不觉心慌意乱，脸上连中了两拳。', '郭靖乘势直上，拍的一记，又在他天灵盖上击了一掌，那童子立时昏晕过去。', '郭靖提足将他拨入路旁草丛，回进房去，打火点亮蜡烛，见那简管家倒在地下，兀自昏晕。', '郭靖暗骂自己胡涂：“那童儿刚才从哪四个瓶罐里取药，我可全没留意，现今怎知这四味药放在哪里”但见瓶罐上面画的都是些弯弯曲曲的符号，竟无一个文字，心下好生为难：“记得他是站在这里拿的，我且把这个角落里的数十罐药每样都拿些，回头请王道长选出来就是。', '取过一叠白纸，每样药材都包了一包，生怕刚才简管家叫喊时被人听见，心里一急，包得更加慢了。', '好容易在每个药瓶中都取了药包好，揣在怀里，大功告成，心下欢喜，回过身来，不提防手肘在旁边的大竹篓上一撞。', '那竹篓横跌翻倒，盖子落下，蓦地呼噜一声，窜出一条殷红如血的大蛇，猛向他脸上扑来。', '郭靖大吃一惊，急忙向后纵开，只见那蛇身子有小碗粗细，半身尚在篓中，不知其长几何，最怪的是通体朱红，蛇头忽伸忽缩，蛇口中伸出一条分叉的舌头，不住向他摇动。']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 已知if '\\u4e00' <= char <= '\\u9fa5': 可以判断一个char是否是中文字\n",
        "\n",
        "# 我希望实现一个函数，这个函数的输入是两个list of string, 长度为M的query 和 长度为N的datas\n",
        "\n",
        "# 输出是一个M*N的numpy float数组 recalls\n",
        "\n",
        "# 先计算freqs[m][n] 表示query的第m句中的每一个中文字，是否在datas[n]中是否出现，如果出现，则freqs[m][n]加一\n",
        "\n",
        "# 然后计算recalls[m][n]是freqs[m][n]除掉 query[m]中所有中文字的个数\n",
        "\n",
        "def compute_char_recall(query, datas):\n",
        "    M = len(query)\n",
        "    N = len(datas)\n",
        "\n",
        "    freqs = np.zeros((M, N), dtype=int)\n",
        "\n",
        "    for m in range(M):\n",
        "        q_chars = set()\n",
        "        for char in query[m]:\n",
        "            if '\\u4e00' <= char <= '\\u9fa5':\n",
        "                q_chars.add(char)\n",
        "\n",
        "        for n in range(N):\n",
        "            for char in q_chars:\n",
        "                if char in datas[n]:\n",
        "                    freqs[m][n] += 1\n",
        "\n",
        "    query_chars_count = [len(set(char for char in sent if '\\u4e00'<= char <= '\\u9fa5'))\n",
        "                         for sent in query]\n",
        "\n",
        "    recalls = freqs / np.array(query_chars_count)[:, None]\n",
        "\n",
        "    return recalls"
      ],
      "metadata": {
        "id": "uRdwFW3toLBA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 可视化用代码，匹配时可以不运行"
      ],
      "metadata": {
        "id": "Od64q6y7ofTi"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import plotly.express as px\n",
        "\n",
        "s = compute_char_recall(chunk_sum, lines)\n",
        "\n",
        "fig = px.imshow(s,\n",
        "                labels=dict(x=\"Line\", y=\"Summary\", color=\"Recall\"),\n",
        "                x=list(range(len(lines))),\n",
        "                y=list(range(len(chunk_sum))),\n",
        "                color_continuous_scale='YlOrRd')\n",
        "\n",
        "fig.update_xaxes(side=\"top\")\n",
        "\n",
        "fig.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 542
        },
        "id": "7mH_PiUDvezU",
        "outputId": "1aae92f0-d46f-48d0-c095-36c92083e47c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html>\n",
              "<head><meta charset=\"utf-8\" /></head>\n",
              "<body>\n",
              "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
              "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.24.1.min.js\"></script>                <div id=\"65061df8-cee2-488e-81c7-d17491347a1d\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"65061df8-cee2-488e-81c7-d17491347a1d\")) {                    Plotly.newPlot(                        \"65061df8-cee2-488e-81c7-d17491347a1d\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19],\"y\":[0,1,2,3,4,5,6],\"z\":[[0.5333333333333333,0.06666666666666667,0.2,0.06666666666666667,0.26666666666666666,0.2,0.2,0.13333333333333333,0.13333333333333333,0.06666666666666667,0.0,0.26666666666666666,0.0,0.13333333333333333,0.3333333333333333,0.4666666666666667,0.2,0.2,0.0,0.2],[0.29411764705882354,0.17647058823529413,0.11764705882352941,0.17647058823529413,0.7058823529411765,0.4117647058823529,0.11764705882352941,0.29411764705882354,0.35294117647058826,0.058823529411764705,0.0,0.23529411764705882,0.11764705882352941,0.23529411764705882,0.35294117647058826,0.29411764705882354,0.23529411764705882,0.11764705882352941,0.11764705882352941,0.17647058823529413],[0.35,0.2,0.1,0.1,0.35,0.65,0.15,0.15,0.15,0.15,0.15,0.2,0.2,0.25,0.15,0.2,0.15,0.2,0.05,0.25],[0.2727272727272727,0.2727272727272727,0.18181818181818182,0.2727272727272727,0.5454545454545454,0.5454545454545454,0.2727272727272727,0.36363636363636365,0.45454545454545453,0.09090909090909091,0.0,0.36363636363636365,0.0,0.2727272727272727,0.5454545454545454,0.18181818181818182,0.2727272727272727,0.0,0.0,0.18181818181818182],[0.26666666666666666,0.06666666666666667,0.13333333333333333,0.0,0.26666666666666666,0.4,0.2,0.13333333333333333,0.13333333333333333,0.2,0.13333333333333333,0.2,0.13333333333333333,0.3333333333333333,0.13333333333333333,0.26666666666666666,0.0,0.06666666666666667,0.13333333333333333,0.3333333333333333],[0.38461538461538464,0.07692307692307693,0.15384615384615385,0.0,0.3076923076923077,0.38461538461538464,0.23076923076923078,0.23076923076923078,0.23076923076923078,0.23076923076923078,0.15384615384615385,0.15384615384615385,0.23076923076923078,0.5384615384615384,0.5384615384615384,0.3076923076923077,0.0,0.15384615384615385,0.07692307692307693,0.46153846153846156],[0.38095238095238093,0.14285714285714285,0.14285714285714285,0.09523809523809523,0.2857142857142857,0.3333333333333333,0.14285714285714285,0.14285714285714285,0.14285714285714285,0.047619047619047616,0.09523809523809523,0.23809523809523808,0.09523809523809523,0.14285714285714285,0.14285714285714285,0.38095238095238093,0.23809523809523808,0.42857142857142855,0.38095238095238093,0.47619047619047616]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Line: %{x}\\u003cbr\\u003eSummary: %{y}\\u003cbr\\u003eRecall: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Line\"},\"side\":\"top\"},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Summary\"}},\"coloraxis\":{\"colorbar\":{\"title\":{\"text\":\"Recall\"}},\"colorscale\":[[0.0,\"rgb(255,255,204)\"],[0.125,\"rgb(255,237,160)\"],[0.25,\"rgb(254,217,118)\"],[0.375,\"rgb(254,178,76)\"],[0.5,\"rgb(253,141,60)\"],[0.625,\"rgb(252,78,42)\"],[0.75,\"rgb(227,26,28)\"],[0.875,\"rgb(189,0,38)\"],[1.0,\"rgb(128,0,38)\"]]},\"margin\":{\"t\":60}},                        {\"responsive\": true}                    ).then(function(){\n",
              "                            \n",
              "var gd = document.getElementById('65061df8-cee2-488e-81c7-d17491347a1d');\n",
              "var x = new MutationObserver(function (mutations, observer) {{\n",
              "        var display = window.getComputedStyle(gd).display;\n",
              "        if (!display || display === 'none') {{\n",
              "            console.log([gd, 'removed!']);\n",
              "            Plotly.purge(gd);\n",
              "            observer.disconnect();\n",
              "        }}\n",
              "}});\n",
              "\n",
              "// Listen for the removal of the full notebook cells\n",
              "var notebookContainer = gd.closest('#notebook-container');\n",
              "if (notebookContainer) {{\n",
              "    x.observe(notebookContainer, {childList: true});\n",
              "}}\n",
              "\n",
              "// Listen for the clearing of the current output cell\n",
              "var outputEl = gd.closest('.output');\n",
              "if (outputEl) {{\n",
              "    x.observe(outputEl, {childList: true});\n",
              "}}\n",
              "\n",
              "                        })                };                            </script>        </div>\n",
              "</body>\n",
              "</html>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 总结匹配"
      ],
      "metadata": {
        "id": "rjVmvuHz_M1h"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# import copy\n",
        "def summary2line(chunk_sum, lines):\n",
        "\n",
        "\n",
        "  s = compute_char_recall(chunk_sum, lines)\n",
        "\n",
        "  color_map = {}\n",
        "  ans_Q = {}\n",
        "\n",
        "  ans_div = {}\n",
        "\n",
        "  flags = {}\n",
        "\n",
        "  M = len( chunk_sum )\n",
        "  N = len( lines )\n",
        "\n",
        "\n",
        "  for n in range(0, N):\n",
        "      if n==0:\n",
        "          ans_Q[ (0,0) ] = s[0,0]\n",
        "          ans_div[ (0,0 ) ] = []\n",
        "      else:\n",
        "          ans_Q[ (0,n) ] = ans_Q[ (0,n-1) ] + s[0,n]\n",
        "          ans_div[ (0,n) ] = []\n",
        "\n",
        "  for m in range(1,M):\n",
        "      ans_Q[(m,m)] = ans_Q[(m-1,m-1)] + s[m,m]\n",
        "      ans_div[ (m,m) ] =  ans_div[ (m-1,m-1) ].copy()\n",
        "      ans_div[ (m,m) ].append(m)\n",
        "\n",
        "\n",
        "  def find_Q( m , n ):\n",
        "      # print(m,n)\n",
        "\n",
        "      if m < 0 or n < 0:\n",
        "          print('error out bound', m , ' ' , n )\n",
        "          return 0, []\n",
        "\n",
        "      if (m,n) in ans_Q.keys():\n",
        "          return ans_Q[(m,n)], ans_div[(m,n)]\n",
        "\n",
        "      if (m,n) in color_map.keys():\n",
        "          print('error repeated quest ', m , ' ', n )\n",
        "          return 0, []\n",
        "      else:\n",
        "          color_map[(m,n)] = 1\n",
        "\n",
        "      current_div = []\n",
        "\n",
        "      left, left_div = find_Q( m, n-1 )\n",
        "      right, right_div = find_Q( m-1, n-1 )\n",
        "\n",
        "      if left > right:\n",
        "          ans = left + s[m][n]\n",
        "          flags[(m,n)] = False\n",
        "          current_div = left_div\n",
        "\n",
        "      else:\n",
        "          ans = right + s[m][n]\n",
        "          flags[(m,n)] = True\n",
        "          current_div = right_div.copy()\n",
        "          current_div.append(n-1)\n",
        "\n",
        "      # ans = max(  , ) + s[m][n]\n",
        "\n",
        "      ans_Q[(m,n)] = ans\n",
        "      ans_div[(m,n)] = current_div.copy()\n",
        "\n",
        "      return ans, current_div\n",
        "\n",
        "  # print(find_Q(0,5))\n",
        "  # print(find_Q(M-1,N-1))\n",
        "\n",
        "  score, divs = find_Q(M-1,N-1)\n",
        "  divs.append(N-1)\n",
        "\n",
        "  return score, divs\n",
        "\n",
        "score, divs = summary2line(chunk_sum, lines)\n",
        "print(score, divs)\n",
        "\n",
        "last = 0\n",
        "\n",
        "# divs.append(N-1)\n",
        "\n",
        "for i,div in enumerate(divs):\n",
        "    print('\\n###')\n",
        "    print(chunk_sum[i])\n",
        "    print('\\n')\n",
        "\n",
        "    for j in range(last, div):\n",
        "        print(lines[j])\n",
        "        last = div"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bHd_rxcNgrc5",
        "outputId": "aafbc8b7-889e-4ee3-fc2c-b5fd1fd793c2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "7.196500558265264 [1, 4, 5, 8, 11, 14, 19]\n",
            "\n",
            "###\n",
            "郭靖进入房间，发现满满都是药材和药瓶\n",
            "\n",
            "\n",
            "郭靖一踏进房，便觉药气冲鼻，又见桌上、榻上、地下，到处放满了诸般药材，以及大大小小的瓶儿、罐儿、缸儿、钵儿，看来梁子翁喜爱调弄丹药，虽在客中，也不放下这些家伙。\n",
            "\n",
            "###\n",
            "简管家故意将门关上，喊贼引起郭靖的注意\n",
            "\n",
            "\n",
            "那小童显也熟习药性，取了四味药，用白纸分别包了，交给简管家。\n",
            "郭靖伸手接过，转身出房。\n",
            "他药已到手，不再看住简管家。\n",
            "\n",
            "###\n",
            "郭靖推门不开，青衣童子夺走了药包并扔进池塘\n",
            "\n",
            "\n",
            "不料这管家甚是狡猾，出房时故意落后，待郭靖与那小童一出门，立时将门关上，撑上门闩，大声叫喊：“有贼啊，有贼啊”郭靖一怔，转身推门，那门甚是坚实，一时推之不开。\n",
            "\n",
            "###\n",
            "郭靖打破门闩，击败简管家\n",
            "\n",
            "\n",
            "那青衣童子年纪虽小，却机怜异常，听得简管家叫喊，知道不妙，乘郭靖使力推门之际，夹手抢过他手中那四包药，往旁边池塘中一丢。\n",
            "郭靖击出两掌，居然都给他闪避开去。\n",
            "郭靖又惊又怒，双掌按在门上，运起内力，喀喇一响，门闩立时崩断。\n",
            "\n",
            "###\n",
            "郭靖追赶青衣童子，展开激烈的战斗\n",
            "\n",
            "\n",
            "他抢进门去，一拳击在简管家下颚之上，颚骨登时碎裂，哪里还能做声幸好梁子翁性喜僻静，居处指定要与别的房舍远离，那简管家这几下叫唤，倒无旁人听到。\n",
            "他回身出门，见那童子已奔在数丈之外，急忙提气纵身，霎时间已追到身后，伸手往他后领抓落。\n",
            "那童子听得脑后风响，身子一挫，右腿横扫，身手竟自不弱。\n",
            "\n",
            "###\n",
            "郭靖击晕童子，将其藏在草丛中\n",
            "\n",
            "\n",
            "郭靖知道只要给他声张出来，不但药物不能得手，而且黄蓉与自己尚有性命之忧，下手更不容情，钩、拿、抓、打，招招是分筋错骨手的狠辣家数。\n",
            "那童子跟着梁子翁，到处受人尊敬，从未遇过强敌，这时不觉心慌意乱，脸上连中了两拳。\n",
            "郭靖乘势直上，拍的一记，又在他天灵盖上击了一掌，那童子立时昏晕过去。\n",
            "\n",
            "###\n",
            "郭靖包好药材，不小心撞倒竹篓，释放出一条红色蛇\n",
            "\n",
            "\n",
            "郭靖提足将他拨入路旁草丛，回进房去，打火点亮蜡烛，见那简管家倒在地下，兀自昏晕。\n",
            "郭靖暗骂自己胡涂：“那童儿刚才从哪四个瓶罐里取药，我可全没留意，现今怎知这四味药放在哪里”但见瓶罐上面画的都是些弯弯曲曲的符号，竟无一个文字，心下好生为难：“记得他是站在这里拿的，我且把这个角落里的数十罐药每样都拿些，回头请王道长选出来就是。\n",
            "取过一叠白纸，每样药材都包了一包，生怕刚才简管家叫喊时被人听见，心里一急，包得更加慢了。\n",
            "好容易在每个药瓶中都取了药包好，揣在怀里，大功告成，心下欢喜，回过身来，不提防手肘在旁边的大竹篓上一撞。\n",
            "那竹篓横跌翻倒，盖子落下，蓦地呼噜一声，窜出一条殷红如血的大蛇，猛向他脸上扑来。\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 可视化用代码，匹配时可以不运行"
      ],
      "metadata": {
        "id": "krqUfFG8sRoV"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import plotly.express as px\n",
        "\n",
        "s_dialogue = compute_char_recall(dia_texts, lines)\n",
        "\n",
        "fig = px.imshow(s_dialogue,\n",
        "                labels=dict(x=\"Line\", y=\"Dialogue\", color=\"Recall\"),\n",
        "                x=list(range(len(lines))),\n",
        "                y=list(range(len(dia_texts))),\n",
        "                color_continuous_scale='YlOrRd')\n",
        "\n",
        "fig.update_xaxes(side=\"top\")\n",
        "\n",
        "fig.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 542
        },
        "id": "npP0xTuVsA-Y",
        "outputId": "b20c28ec-5f7e-45af-a36c-3423ab271de3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html>\n",
              "<head><meta charset=\"utf-8\" /></head>\n",
              "<body>\n",
              "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
              "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.24.1.min.js\"></script>                <div id=\"0e704e9c-43d6-4fdd-a015-26324cd83d55\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"0e704e9c-43d6-4fdd-a015-26324cd83d55\")) {                    Plotly.newPlot(                        \"0e704e9c-43d6-4fdd-a015-26324cd83d55\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19],\"y\":[0,1,2,3,4,5,6,7,8],\"z\":[[0.75,0.3157894736842105,0.039473684210526314,0.09210526315789473,0.17105263157894737,0.21052631578947367,0.039473684210526314,0.07894736842105263,0.2631578947368421,0.09210526315789473,0.06578947368421052,0.14473684210526316,0.17105263157894737,0.14473684210526316,0.15789473684210525,0.27631578947368424,0.15789473684210525,0.19736842105263158,0.11842105263157894,0.17105263157894737],[0.16161616161616163,0.10101010101010101,0.10101010101010101,0.12121212121212122,0.46464646464646464,0.5050505050505051,0.15151515151515152,0.10101010101010101,0.23232323232323232,0.16161616161616163,0.10101010101010101,0.1717171717171717,0.12121212121212122,0.15151515151515152,0.1111111111111111,0.18181818181818182,0.13131313131313133,0.13131313131313133,0.09090909090909091,0.1919191919191919],[0.25,0.0,0.08333333333333333,0.0,0.3333333333333333,0.20833333333333334,0.125,1.0,0.20833333333333334,0.125,0.08333333333333333,0.08333333333333333,0.08333333333333333,0.375,0.125,0.20833333333333334,0.08333333333333333,0.125,0.08333333333333333,0.20833333333333334],[0.2542372881355932,0.1016949152542373,0.01694915254237288,0.0847457627118644,0.22033898305084745,0.22033898305084745,0.06779661016949153,0.0847457627118644,1.0,0.13559322033898305,0.06779661016949153,0.1864406779661017,0.1864406779661017,0.1694915254237288,0.2033898305084746,0.2033898305084746,0.15254237288135594,0.15254237288135594,0.15254237288135594,0.11864406779661017],[0.15555555555555556,0.07777777777777778,0.06666666666666667,0.07777777777777778,0.2,0.2,0.05555555555555555,0.07777777777777778,0.2111111111111111,0.36666666666666664,0.23333333333333334,0.5666666666666667,0.06666666666666667,0.1111111111111111,0.13333333333333333,0.28888888888888886,0.1,0.13333333333333333,0.12222222222222222,0.25555555555555554],[0.2804878048780488,0.07317073170731707,0.04878048780487805,0.07317073170731707,0.18292682926829268,0.18292682926829268,0.08536585365853659,0.10975609756097561,0.3048780487804878,0.12195121951219512,0.07317073170731707,0.10975609756097561,0.4146341463414634,0.34146341463414637,0.4268292682926829,0.21951219512195122,0.12195121951219512,0.15853658536585366,0.13414634146341464,0.13414634146341464],[0.18269230769230768,0.125,0.038461538461538464,0.04807692307692308,0.15384615384615385,0.18269230769230768,0.04807692307692308,0.057692307692307696,0.18269230769230768,0.10576923076923077,0.0673076923076923,0.19230769230769232,0.10576923076923077,0.125,0.11538461538461539,0.7980769230769231,0.3269230769230769,0.18269230769230768,0.08653846153846154,0.1346153846153846],[0.22388059701492538,0.07462686567164178,0.05970149253731343,0.05970149253731343,0.13432835820895522,0.1791044776119403,0.04477611940298507,0.04477611940298507,0.208955223880597,0.14925373134328357,0.1044776119402985,0.14925373134328357,0.13432835820895522,0.14925373134328357,0.13432835820895522,0.29850746268656714,0.14925373134328357,0.6119402985074627,0.5074626865671642,0.23880597014925373],[0.19642857142857142,0.05357142857142857,0.08928571428571429,0.05357142857142857,0.23214285714285715,0.17857142857142858,0.08928571428571429,0.08928571428571429,0.125,0.21428571428571427,0.10714285714285714,0.21428571428571427,0.07142857142857142,0.14285714285714285,0.10714285714285714,0.23214285714285715,0.05357142857142857,0.14285714285714285,0.21428571428571427,1.0]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Line: %{x}\\u003cbr\\u003eDialogue: %{y}\\u003cbr\\u003eRecall: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Line\"},\"side\":\"top\"},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Dialogue\"}},\"coloraxis\":{\"colorbar\":{\"title\":{\"text\":\"Recall\"}},\"colorscale\":[[0.0,\"rgb(255,255,204)\"],[0.125,\"rgb(255,237,160)\"],[0.25,\"rgb(254,217,118)\"],[0.375,\"rgb(254,178,76)\"],[0.5,\"rgb(253,141,60)\"],[0.625,\"rgb(252,78,42)\"],[0.75,\"rgb(227,26,28)\"],[0.875,\"rgb(189,0,38)\"],[1.0,\"rgb(128,0,38)\"]]},\"margin\":{\"t\":60}},                        {\"responsive\": true}                    ).then(function(){\n",
              "                            \n",
              "var gd = document.getElementById('0e704e9c-43d6-4fdd-a015-26324cd83d55');\n",
              "var x = new MutationObserver(function (mutations, observer) {{\n",
              "        var display = window.getComputedStyle(gd).display;\n",
              "        if (!display || display === 'none') {{\n",
              "            console.log([gd, 'removed!']);\n",
              "            Plotly.purge(gd);\n",
              "            observer.disconnect();\n",
              "        }}\n",
              "}});\n",
              "\n",
              "// Listen for the removal of the full notebook cells\n",
              "var notebookContainer = gd.closest('#notebook-container');\n",
              "if (notebookContainer) {{\n",
              "    x.observe(notebookContainer, {childList: true});\n",
              "}}\n",
              "\n",
              "// Listen for the clearing of the current output cell\n",
              "var outputEl = gd.closest('.output');\n",
              "if (outputEl) {{\n",
              "    x.observe(outputEl, {childList: true});\n",
              "}}\n",
              "\n",
              "                        })                };                            </script>        </div>\n",
              "</body>\n",
              "</html>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "s_dialogue 存储了一个M*N的np array\n",
        "\n",
        "我现在希望实现一个python程序，能够找到一个长度为M的顺序子序列 a_0, ... , a_m-1\n",
        "\n",
        "使得s_dialogue[ i ][ a_i ] 之和最大\n",
        "\n",
        "输出a_0, ... , a_{m-1}的值\n",
        "\n",
        "用动态规划算法，python为我实现\n"
      ],
      "metadata": {
        "id": "6MC3lZvmlvj_"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 对话匹配"
      ],
      "metadata": {
        "id": "HRXExnjN_RUa"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def dialogue2line(dia_texts, lines):\n",
        "    s_dialogue = compute_char_recall(dia_texts, lines)\n",
        "\n",
        "    M, N = s_dialogue.shape\n",
        "    if M==0 or N==0:\n",
        "      return []\n",
        "    dp = np.zeros((M, N))\n",
        "    dp[0] = s_dialogue[0]\n",
        "    prev_indices = np.zeros((M, N), dtype=int)\n",
        "    for i in range(1, M):\n",
        "        for j in range(N):\n",
        "            max_prev_index = np.argmax(dp[i-1])\n",
        "            dp[i][j] = dp[i-1][max_prev_index] + s_dialogue[i][j]\n",
        "            prev_indices[i][j] = max_prev_index\n",
        "\n",
        "    max_end_index = np.argmax(dp[-1])\n",
        "    sequence = []\n",
        "    for i in range(M-1, -1, -1):\n",
        "        sequence.append(max_end_index)\n",
        "        max_end_index = prev_indices[i][max_end_index]\n",
        "    sequence.reverse()\n",
        "\n",
        "    return sequence\n",
        "\n",
        "seq = dialogue2line(dia_texts, lines)\n",
        "print(seq)\n",
        "\n",
        "for i,id in enumerate(seq):\n",
        "    print('\\n###')\n",
        "    print(dia_texts[i])\n",
        "    print(lines[id])\n",
        "    print('\\n')\n",
        "\n",
        "    for j in range(last, div):\n",
        "        print(lines[j])\n",
        "        last = div"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qFGcMmM8luWH",
        "outputId": "0e34a159-b763-4c2c-b70f-2e25f6289d1a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[0, 5, 7, 8, 11, 14, 15, 17, 19]\n",
            "\n",
            "###\n",
            "郭靖一踏进房，便觉药气冲鼻，又见桌上、榻上、地下，到处放满了诸般药材，以及大大小小的瓶儿、罐儿、缸儿、钵儿，看来梁子翁喜爱调弄丹药，虽在客中，也不放下这些家伙。那小童显也熟习药性，取了四味药，用白纸分别包了，交给简管家。\n",
            "郭靖一踏进房，便觉药气冲鼻，又见桌上、榻上、地下，到处放满了诸般药材，以及大大小小的瓶儿、罐儿、缸儿、钵儿，看来梁子翁喜爱调弄丹药，虽在客中，也不放下这些家伙。\n",
            "\n",
            "\n",
            "\n",
            "###\n",
            "郭靖伸手接过，转身出房。他药已到手，不再看住简管家。不料这管家甚是狡猾，出房时故意落后，待郭靖与那小童一出门，立时将门关上，撑上门闩，大声叫喊：“有贼啊，有贼啊”郭靖一怔，转身推门，那门甚是坚实，一时推之不开。那青衣童子年纪虽小，却机怜异常，听得简管家叫喊，知道不妙，乘郭靖使力推门之际，夹手抢过他手中那四包药，往旁边池塘中一丢。郭靖击出两掌，居然都给他闪避开去。\n",
            "那青衣童子年纪虽小，却机怜异常，听得简管家叫喊，知道不妙，乘郭靖使力推门之际，夹手抢过他手中那四包药，往旁边池塘中一丢。\n",
            "\n",
            "\n",
            "\n",
            "###\n",
            "郭靖又惊又怒，双掌按在门上，运起内力，喀喇一响，门闩立时崩断。\n",
            "郭靖又惊又怒，双掌按在门上，运起内力，喀喇一响，门闩立时崩断。\n",
            "\n",
            "\n",
            "\n",
            "###\n",
            "他抢进门去，一拳击在简管家下颚之上，颚骨登时碎裂，哪里还能做声幸好梁子翁性喜僻静，居处指定要与别的房舍远离，那简管家这几下叫唤，倒无旁人听到。\n",
            "他抢进门去，一拳击在简管家下颚之上，颚骨登时碎裂，哪里还能做声幸好梁子翁性喜僻静，居处指定要与别的房舍远离，那简管家这几下叫唤，倒无旁人听到。\n",
            "\n",
            "\n",
            "\n",
            "###\n",
            "他回身出门，见那童子已奔在数丈之外，急忙提气纵身，霎时间已追到身后，伸手往他后领抓落。那童子听得脑后风响，身子一挫，右腿横扫，身手竟自不弱。郭靖知道只要给他声张出来，不但药物不能得手，而且黄蓉与自己尚有性命之忧，下手更不容情，钩、拿、抓、打，招招是分筋错骨手的狠辣家数。\n",
            "郭靖知道只要给他声张出来，不但药物不能得手，而且黄蓉与自己尚有性命之忧，下手更不容情，钩、拿、抓、打，招招是分筋错骨手的狠辣家数。\n",
            "\n",
            "\n",
            "\n",
            "###\n",
            "那童子跟着梁子翁，到处受人尊敬，从未遇过强敌，这时不觉心慌意乱，脸上连中了两拳。郭靖乘势直上，拍的一记，又在他天灵盖上击了一掌，那童子立时昏晕过去。郭靖提足将他拨入路旁草丛，回进房去，打火点亮蜡烛，见那简管家倒在地下，兀自昏晕。\n",
            "郭靖提足将他拨入路旁草丛，回进房去，打火点亮蜡烛，见那简管家倒在地下，兀自昏晕。\n",
            "\n",
            "\n",
            "\n",
            "###\n",
            "郭靖暗骂自己胡涂：“那童儿刚才从哪四个瓶罐里取药，我可全没留意，现今怎知这四味药放在哪里”但见瓶罐上面画的都是些弯弯曲曲的符号，竟无一个文字，心下好生为难：“记得他是站在这里拿的，我且把这个角落里的数十罐药每样都拿些，回头请王道长选出来就是。”取过一叠白纸，每样药材都包了一包，生怕刚才简管家叫喊时被人听见，心里一急，包得更加慢了。\n",
            "郭靖暗骂自己胡涂：“那童儿刚才从哪四个瓶罐里取药，我可全没留意，现今怎知这四味药放在哪里”但见瓶罐上面画的都是些弯弯曲曲的符号，竟无一个文字，心下好生为难：“记得他是站在这里拿的，我且把这个角落里的数十罐药每样都拿些，回头请王道长选出来就是。\n",
            "\n",
            "\n",
            "\n",
            "###\n",
            "好容易在每个药瓶中都取了药包好，揣在怀里，大功告成，心下欢喜，回过身来，不提防手肘在旁边的大竹篓上一撞。那竹篓横跌翻倒，盖子落下，蓦地呼噜一声，窜出一条殷红如血的大蛇，猛向他脸上扑来。\n",
            "好容易在每个药瓶中都取了药包好，揣在怀里，大功告成，心下欢喜，回过身来，不提防手肘在旁边的大竹篓上一撞。\n",
            "\n",
            "\n",
            "\n",
            "###\n",
            "郭靖大吃一惊，急忙向后纵开，只见那蛇身子有小碗粗细，半身尚在篓中，不知其长几何，最怪的是通体朱红，蛇头忽伸忽缩，蛇口中伸出一条分叉的舌头，不住向他摇动。\n",
            "郭靖大吃一惊，急忙向后纵开，只见那蛇身子有小碗粗细，半身尚在篓中，不知其长几何，最怪的是通体朱红，蛇头忽伸忽缩，蛇口中伸出一条分叉的舌头，不住向他摇动。\n",
            "\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 运行后请查看/content/reorganized_{id}.jsonl处文件效果"
      ],
      "metadata": {
        "id": "rTYP4Fhe_XmK"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "\n",
        "print(len(chunk_sum))\n",
        "print(divs)\n",
        "print(dia_texts)\n",
        "print(seq)\n",
        "\n",
        "\n",
        "# string_indices = [10, 11]\n",
        "def jsonl_sorted(chunk_sum, divs, dia_texts, seq):\n",
        "\n",
        "  combined_data = []\n",
        "  combined_text = \"\"\n",
        "  for index in sorted(seq + divs):\n",
        "      # print(index)\n",
        "      if index in seq:\n",
        "\n",
        "          combined_data.append({\n",
        "              \"role\" : dialogues[seq.index(index)][\"role\"],\n",
        "              'text': dialogues[seq.index(index)][\"dialogue\"],\n",
        "              'if_scene': False\n",
        "          })\n",
        "          combined_text = combined_text + dialogues[seq.index(index)][\"role\"] + \":\" + dialogues[seq.index(index)][\"dialogue\"] +\"\\n\"\n",
        "          seq[seq.index(index)] = -1\n",
        "      if index in divs:\n",
        "          combined_data.append({\n",
        "              \"role\" : \"scene\" ,\n",
        "              'text': chunk_sum[divs.index(index)],\n",
        "              'if_scene': True\n",
        "          })\n",
        "          combined_text = combined_text + \"scene\" + \":\" + chunk_sum[divs.index(index)] +\"\\n\"\n",
        "          divs[divs.index(index)]=-1\n",
        "\n",
        "  return combined_data, combined_text\n",
        "\n",
        "combined_data, combined_text = jsonl_sorted(chunk_sum, divs.copy(), dia_texts, seq.copy())\n",
        "print(combined_data)\n",
        "# 打开文件，以写入模式（\"w\"）打开\n",
        "with open(f\"/content/reorganized_{id}.jsonl\", \"w\", encoding=\"utf-8\") as file:\n",
        "    # 遍历数据列表中的每个字典\n",
        "    for record in combined_data:\n",
        "        # 将字典转换为JSON格式的字符串\n",
        "        json_record = json.dumps(record, ensure_ascii=False)\n",
        "        # 将转换后的JSON字符串写入文件，并添加换行符\n",
        "        file.write(json_record + \"\\n\")\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "O-85AvjythtR",
        "outputId": "5d298f4e-d163-4640-bf49-aa7458476f48"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "7\n",
            "[1, 4, 5, 8, 11, 14, 19]\n",
            "['郭靖一踏进房，便觉药气冲鼻，又见桌上、榻上、地下，到处放满了诸般药材，以及大大小小的瓶儿、罐儿、缸儿、钵儿，看来梁子翁喜爱调弄丹药，虽在客中，也不放下这些家伙。那小童显也熟习药性，取了四味药，用白纸分别包了，交给简管家。', '郭靖伸手接过，转身出房。他药已到手，不再看住简管家。不料这管家甚是狡猾，出房时故意落后，待郭靖与那小童一出门，立时将门关上，撑上门闩，大声叫喊：“有贼啊，有贼啊”郭靖一怔，转身推门，那门甚是坚实，一时推之不开。那青衣童子年纪虽小，却机怜异常，听得简管家叫喊，知道不妙，乘郭靖使力推门之际，夹手抢过他手中那四包药，往旁边池塘中一丢。郭靖击出两掌，居然都给他闪避开去。', '郭靖又惊又怒，双掌按在门上，运起内力，喀喇一响，门闩立时崩断。', '他抢进门去，一拳击在简管家下颚之上，颚骨登时碎裂，哪里还能做声幸好梁子翁性喜僻静，居处指定要与别的房舍远离，那简管家这几下叫唤，倒无旁人听到。', '他回身出门，见那童子已奔在数丈之外，急忙提气纵身，霎时间已追到身后，伸手往他后领抓落。那童子听得脑后风响，身子一挫，右腿横扫，身手竟自不弱。郭靖知道只要给他声张出来，不但药物不能得手，而且黄蓉与自己尚有性命之忧，下手更不容情，钩、拿、抓、打，招招是分筋错骨手的狠辣家数。', '那童子跟着梁子翁，到处受人尊敬，从未遇过强敌，这时不觉心慌意乱，脸上连中了两拳。郭靖乘势直上，拍的一记，又在他天灵盖上击了一掌，那童子立时昏晕过去。郭靖提足将他拨入路旁草丛，回进房去，打火点亮蜡烛，见那简管家倒在地下，兀自昏晕。', '郭靖暗骂自己胡涂：“那童儿刚才从哪四个瓶罐里取药，我可全没留意，现今怎知这四味药放在哪里”但见瓶罐上面画的都是些弯弯曲曲的符号，竟无一个文字，心下好生为难：“记得他是站在这里拿的，我且把这个角落里的数十罐药每样都拿些，回头请王道长选出来就是。”取过一叠白纸，每样药材都包了一包，生怕刚才简管家叫喊时被人听见，心里一急，包得更加慢了。', '好容易在每个药瓶中都取了药包好，揣在怀里，大功告成，心下欢喜，回过身来，不提防手肘在旁边的大竹篓上一撞。那竹篓横跌翻倒，盖子落下，蓦地呼噜一声，窜出一条殷红如血的大蛇，猛向他脸上扑来。', '郭靖大吃一惊，急忙向后纵开，只见那蛇身子有小碗粗细，半身尚在篓中，不知其长几何，最怪的是通体朱红，蛇头忽伸忽缩，蛇口中伸出一条分叉的舌头，不住向他摇动。']\n",
            "[0, 5, 7, 8, 11, 14, 15, 17, 19]\n",
            "[{'role': '郭靖', 'text': '郭靖一踏进房，便觉药气冲鼻，又见桌上、榻上、地下，到处放满了诸般药材，以及大大小小的瓶儿、罐儿、缸儿、钵儿，看来梁子翁喜爱调弄丹药，虽在客中，也不放下这些家伙。那小童显也熟习药性，取了四味药，用白纸分别包了，交给简管家。', 'if_scene': False}, {'role': 'scene', 'text': '郭靖进入房间，发现满满都是药材和药瓶', 'if_scene': True}, {'role': 'scene', 'text': '简管家故意将门关上，喊贼引起郭靖的注意', 'if_scene': True}, {'role': '郭靖', 'text': '郭靖伸手接过，转身出房。他药已到手，不再看住简管家。不料这管家甚是狡猾，出房时故意落后，待郭靖与那小童一出门，立时将门关上，撑上门闩，大声叫喊：“有贼啊，有贼啊”郭靖一怔，转身推门，那门甚是坚实，一时推之不开。那青衣童子年纪虽小，却机怜异常，听得简管家叫喊，知道不妙，乘郭靖使力推门之际，夹手抢过他手中那四包药，往旁边池塘中一丢。郭靖击出两掌，居然都给他闪避开去。', 'if_scene': False}, {'role': 'scene', 'text': '郭靖推门不开，青衣童子夺走了药包并扔进池塘', 'if_scene': True}, {'role': '郭靖', 'text': '郭靖又惊又怒，双掌按在门上，运起内力，喀喇一响，门闩立时崩断。', 'if_scene': False}, {'role': '郭靖', 'text': '他抢进门去，一拳击在简管家下颚之上，颚骨登时碎裂，哪里还能做声幸好梁子翁性喜僻静，居处指定要与别的房舍远离，那简管家这几下叫唤，倒无旁人听到。', 'if_scene': False}, {'role': 'scene', 'text': '郭靖打破门闩，击败简管家', 'if_scene': True}, {'role': '郭靖', 'text': '他回身出门，见那童子已奔在数丈之外，急忙提气纵身，霎时间已追到身后，伸手往他后领抓落。那童子听得脑后风响，身子一挫，右腿横扫，身手竟自不弱。郭靖知道只要给他声张出来，不但药物不能得手，而且黄蓉与自己尚有性命之忧，下手更不容情，钩、拿、抓、打，招招是分筋错骨手的狠辣家数。', 'if_scene': False}, {'role': 'scene', 'text': '郭靖追赶青衣童子，展开激烈的战斗', 'if_scene': True}, {'role': '郭靖', 'text': '那童子跟着梁子翁，到处受人尊敬，从未遇过强敌，这时不觉心慌意乱，脸上连中了两拳。郭靖乘势直上，拍的一记，又在他天灵盖上击了一掌，那童子立时昏晕过去。郭靖提足将他拨入路旁草丛，回进房去，打火点亮蜡烛，见那简管家倒在地下，兀自昏晕。', 'if_scene': False}, {'role': 'scene', 'text': '郭靖击晕童子，将其藏在草丛中', 'if_scene': True}, {'role': '郭靖', 'text': '郭靖暗骂自己胡涂：“那童儿刚才从哪四个瓶罐里取药，我可全没留意，现今怎知这四味药放在哪里”但见瓶罐上面画的都是些弯弯曲曲的符号，竟无一个文字，心下好生为难：“记得他是站在这里拿的，我且把这个角落里的数十罐药每样都拿些，回头请王道长选出来就是。”取过一叠白纸，每样药材都包了一包，生怕刚才简管家叫喊时被人听见，心里一急，包得更加慢了。', 'if_scene': False}, {'role': '郭靖', 'text': '好容易在每个药瓶中都取了药包好，揣在怀里，大功告成，心下欢喜，回过身来，不提防手肘在旁边的大竹篓上一撞。那竹篓横跌翻倒，盖子落下，蓦地呼噜一声，窜出一条殷红如血的大蛇，猛向他脸上扑来。', 'if_scene': False}, {'role': '郭靖', 'text': '郭靖大吃一惊，急忙向后纵开，只见那蛇身子有小碗粗细，半身尚在篓中，不知其长几何，最怪的是通体朱红，蛇头忽伸忽缩，蛇口中伸出一条分叉的舌头，不住向他摇动。', 'if_scene': False}, {'role': 'scene', 'text': '郭靖包好药材，不小心撞倒竹篓，释放出一条红色蛇', 'if_scene': True}]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "清空储存文件夹"
      ],
      "metadata": {
        "id": "AGyxTKaFWEYX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "os.system(f\"rm -rf {save_folder_path}\")\n",
        "os.makedirs(save_folder_path, exist_ok=True)"
      ],
      "metadata": {
        "id": "yKhM_XSfVhzO"
      },
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 以上是手动调整ID版程序，下面是自动化程序，请保证上述单元格除可视化外均已运行，并测试无报错"
      ],
      "metadata": {
        "id": "08_9l-fv-UtF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import os\n",
        "import copy\n",
        "import json\n",
        "from tqdm import tqdm\n",
        "\n",
        "final_jsonl = []\n",
        "final_txt = \"\"\n",
        "\n",
        "for i in tqdm(range(1,len(chunk_text)), desc=\"Processing\", total=len(chunk_text)-1, unit=\"item\"):\n",
        "\n",
        "  try:\n",
        "    # story_name_en = 'shediaoyingxiongzhuan'\n",
        "    raw_text = chunk_text[ i ]\n",
        "\n",
        "    import os\n",
        "\n",
        "    save_folder = f\"/content/drive/MyDrive/{story_name_en}_extract\"\n",
        "\n",
        "    dialoge_file = os.path.join(save_folder, f\"{i}_dialogue.txt\")\n",
        "    summarzie_file = os.path.join(save_folder, f\"{i}_sum.txt\")\n",
        "\n",
        "    chunk_sum = []\n",
        "    unique_chunk_sum = []\n",
        "    if os.path.exists(summarzie_file):\n",
        "        with open(summarzie_file, 'r', encoding='utf-8') as f:\n",
        "            for line in f:\n",
        "                if line.strip().startswith('-'):\n",
        "                    chunk_sum.append(line.strip()[1:].strip())\n",
        "    if os.path.exists(dialoge_file):\n",
        "      with open(dialoge_file, encoding='utf-8') as f:\n",
        "        dialogues = []\n",
        "        for line in f:\n",
        "            dialogue = json.loads(line)\n",
        "            dialogues.append(dialogue)\n",
        "\n",
        "    unique_dialogue = []\n",
        "    for item in dialogues:\n",
        "        if item not in unique_dialogue:\n",
        "            unique_dialogue.append(item)\n",
        "    dia_texts = [data['dialogue'] for data in unique_dialogue]\n",
        "    for item in chunk_sum:\n",
        "        if item not in unique_chunk_sum:\n",
        "            unique_chunk_sum.append(item)\n",
        "\n",
        "    chunk_sum = unique_chunk_sum\n",
        "    dialogues = unique_dialogue\n",
        "    lines, starts, ends = divide_raw2lines(raw_text)\n",
        "    # print(chunk_sum)\n",
        "    # print(lines)\n",
        "    score, divs = summary2line(chunk_sum, lines)  #summary匹配\n",
        "    seq = dialogue2line(dia_texts, lines) #对话匹配\n",
        "    combined_data, combined_text = jsonl_sorted(chunk_sum, divs.copy(), dia_texts, seq.copy())\n",
        "    # 如果需要保存每个chunk的，在此处保存\n",
        "    final_jsonl.append(combined_data)\n",
        "    final_txt = final_txt + combined_text + \"\\n\"\n",
        "  except:\n",
        "    print(\"第\" + str(i) + \"个chunk出错\")\n",
        "    pass\n",
        "with open(save_jsonl_path, \"w\", encoding=\"utf-8\") as file:\n",
        "  # 遍历数据列表中的每个字典\n",
        "  for record in final_jsonl:\n",
        "      # 将字典转换为JSON格式的字符串\n",
        "      json_record = json.dumps(record, ensure_ascii=False)\n",
        "      # 将转换后的JSON字符串写入文件，并添加换行符\n",
        "      file.write(json_record + \"\\n\")\n",
        "with open(save_txt_path, \"w\", encoding=\"utf-8\") as file:\n",
        "  file.write(final_txt)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "63fa133c-d395-49c8-bbf9-c01d364814ff",
        "id": "8hK82pj1-e0D"
      },
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Processing:   1%|▏         | 12/928 [00:00<00:08, 114.00item/s]<ipython-input-20-388c8ce4bd20>:31: RuntimeWarning:\n",
            "\n",
            "invalid value encountered in divide\n",
            "\n",
            "Processing:  41%|████      | 380/928 [00:02<00:04, 132.23item/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "第362个chunk出错\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Processing:  95%|█████████▍| 878/928 [00:07<00:00, 96.02item/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "第861个chunk出错\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Processing: 100%|██████████| 928/928 [00:08<00:00, 113.90item/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# 运行到此处，txt和jsonl文件已保存在/content/drive/MyDrive/reorganized_story_{story_name_en}下"
      ],
      "metadata": {
        "id": "dDHorUK6AA3a"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# 如果你要想从content重新切分，请在开头运行以下代码\n",
        "## 导入原小说，并放在/content下"
      ],
      "metadata": {
        "id": "lAuvl5di_N9h"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# !wget https://gist.githubusercontent.com/zhenghaoz/7ad0948d5835468844abd2db8f67faff/raw/98bd7a3ca7ef308b031a20a0c671b2d2da23ea3e/%25E5%25A4%25A9%25E9%25BE%2599%25E5%2585%25AB%25E9%2583%25A8.txt\n",
        "\n",
        "!wget https://raw.githubusercontent.com/LC1332/Prophet-Andrew-Ng/main/langchain/%E7%AC%91%E5%82%B2%E6%B1%9F%E6%B9%96.txt"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "buYlL4ikSegm",
        "outputId": "6d7e9a7e-3142-499f-897d-42adc6b4516c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2023-09-01 09:56:22--  https://raw.githubusercontent.com/LC1332/Prophet-Andrew-Ng/main/langchain/%E7%AC%91%E5%82%B2%E6%B1%9F%E6%B9%96.txt\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 2969642 (2.8M) [text/plain]\n",
            "Saving to: ‘笑傲江湖.txt’\n",
            "\n",
            "笑傲江湖.txt        100%[===================>]   2.83M  --.-KB/s    in 0.08s   \n",
            "\n",
            "2023-09-01 09:56:23 (34.4 MB/s) - ‘笑傲江湖.txt’ saved [2969642/2969642]\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import re\n",
        "\n",
        "raw_text = open('/content/笑傲江湖.txt', encoding='utf-8').read()\n",
        "\n",
        "chapter_pattern = r'[^\\n]+第\\d+章[^\\n]+\\n'\n",
        "\n",
        "chapters = re.split(chapter_pattern, raw_text)\n",
        "\n",
        "print(len(chapters))\n",
        "\n",
        "# chapters = []\n",
        "# for match in re.finditer(r'\\n第.+?章(.+?)\\n', raw_text):\n",
        "#     title = match.group(1)\n",
        "#     start = match.start()\n",
        "#     end = raw_text.find('\\n第', start+1)\n",
        "#     chapters.append(raw_text[start:end])"
      ],
      "metadata": {
        "id": "U9ArrOeySqSf",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "abb594ab-577d-4125-90a1-b36ee489bbee"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "41\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import tiktoken\n",
        "\n",
        "enc = tiktoken.get_encoding(\"cl100k_base\")"
      ],
      "metadata": {
        "id": "RRonr_bDUZ2M"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 递推切割成接近于1500个token的chunk"
      ],
      "metadata": {
        "id": "E2c7FpZXi6WX"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 章节切割\n",
        "***请修改为你的小说的名字***\n",
        "\n",
        "如果你想导入切分好的chunk，请运行下方导入chunk_list"
      ],
      "metadata": {
        "id": "4v0AGg7JfOst"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def divide_str(s, sep=['\\n', '.', '。']):\n",
        "    mid_len = len(s) // 2  # 中心点位置\n",
        "    best_sep_pos = len(s) + 1  # 最接近中心点的分隔符位置\n",
        "    best_sep = None  # 最接近中心点的分隔符\n",
        "    for curr_sep in sep:\n",
        "        sep_pos = s.rfind(curr_sep, 0, mid_len)  # 从中心点往左找分隔符\n",
        "        if sep_pos > 0 and abs(sep_pos - mid_len) < abs(best_sep_pos -\n",
        "                                                        mid_len):\n",
        "            best_sep_pos = sep_pos\n",
        "            best_sep = curr_sep\n",
        "    if not best_sep:  # 没有找到分隔符\n",
        "        return s, ''\n",
        "    return s[:best_sep_pos + 1], s[best_sep_pos + 1:]\n",
        "\n",
        "\n",
        "def strong_divide(s):\n",
        "    left, right = divide_str(s)\n",
        "\n",
        "    if right != '':\n",
        "        return left, right\n",
        "\n",
        "    whole_sep = ['\\n', '.', '，', '、', ';', ',', '；',\\\n",
        "                 '：', '！', '？', '(', ')', '”', '“', \\\n",
        "                 '’', '‘', '[', ']', '{', '}', '<', '>', \\\n",
        "                 '/', '''\\''', '|', '-', '=', '+', '*', '%', \\\n",
        "               '$', '''#''', '@', '&', '^', '_', '`', '~',\\\n",
        "                 '·', '…']\n",
        "    left, right = divide_str(s, sep=whole_sep)\n",
        "\n",
        "    if right != '':\n",
        "        return left, right\n",
        "\n",
        "    mid_len = len(s) // 2\n",
        "    return s[:mid_len], s[mid_len:]\n"
      ],
      "metadata": {
        "id": "MW5ylWoSUan0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "max_token_len = 1500\n",
        "chunk_text = []\n",
        "\n",
        "\n",
        "flag = True\n",
        "\n",
        "for chapter in chapters:\n",
        "\n",
        "    split_text = chapter.split('\\n')\n",
        "\n",
        "    curr_len = 0\n",
        "    curr_chunk = ''\n",
        "\n",
        "    tmp = []\n",
        "\n",
        "    for line in split_text:\n",
        "        line_len = len(enc.encode( line ))\n",
        "\n",
        "        if line_len <= max_token_len - 5:\n",
        "            tmp.append(line)\n",
        "        else:\n",
        "            # print('divide line with length = ', line_len)\n",
        "            path = [line]\n",
        "            tmp_res = []\n",
        "\n",
        "            while path:\n",
        "                my_str = path.pop()\n",
        "                left, right = strong_divide(my_str)\n",
        "\n",
        "                len_left = len(enc.encode( left ))\n",
        "                len_right = len(enc.encode( right ))\n",
        "\n",
        "                if len_left > max_token_len - 15:\n",
        "                    path.append(left)\n",
        "                else:\n",
        "                    tmp_res.append(left)\n",
        "\n",
        "                if len_right > max_token_len - 15:\n",
        "                    path.append(right)\n",
        "                else:\n",
        "                    tmp_res.append(right)\n",
        "\n",
        "            for line in tmp_res:\n",
        "                tmp.append(line)\n",
        "\n",
        "    split_text = tmp\n",
        "\n",
        "    for line in split_text:\n",
        "        line_len = len(enc.encode( line ))\n",
        "\n",
        "        if line_len > max_token_len:\n",
        "            print('warning line_len = ', line_len)\n",
        "\n",
        "        if curr_len + line_len <= max_token_len:\n",
        "            curr_chunk += line\n",
        "            curr_chunk += '\\n'\n",
        "            curr_len += line_len\n",
        "            curr_len += 1\n",
        "        else:\n",
        "            chunk_text.append(curr_chunk)\n",
        "            curr_chunk = line\n",
        "            curr_len = line_len\n",
        "\n",
        "    if curr_chunk:\n",
        "        chunk_text.append(curr_chunk)\n",
        "\n",
        "    # break\n",
        "\n",
        "print(len(chunk_text))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6ZVBixjzUcAB",
        "outputId": "e5b87645-0afe-4780-bbf2-5974a210cf87"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "976\n"
          ]
        }
      ]
    }
  ]
}
